{
 "cells": [
  {
   "cell_type": "code",
   "id": "b240e235-156b-43ab-84f3-1431ea991cd2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-07T15:26:11.397282Z",
     "start_time": "2024-09-07T15:26:10.782274Z"
    }
   },
   "source": [
    "import jieba  \n",
    "from sklearn.feature_extraction.text import TfidfVectorizer  \n",
    "from sklearn.ensemble import RandomForestClassifier \n",
    "from sklearn.model_selection import train_test_split  \n",
    "from sklearn.metrics import classification_report  "
   ],
   "outputs": [],
   "execution_count": 1
  },
  {
   "cell_type": "code",
   "id": "9e015f4a-7cd8-4c44-882e-61737d35f848",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-07T15:26:12.435066Z",
     "start_time": "2024-09-07T15:26:11.995723Z"
    }
   },
   "source": [
    "import pandas as pd  \n",
    "def load_data(file_path):  \n",
    "    return pd.read_csv(file_path) "
   ],
   "outputs": [],
   "execution_count": 2
  },
  {
   "cell_type": "code",
   "id": "bab42102-e491-47d4-a601-ef52716f01a8",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-07T15:26:14.224660Z",
     "start_time": "2024-09-07T15:26:14.162682Z"
    }
   },
   "source": [
    "data = load_data('Detect_data/train-data-all-0907-001-utf8.csv')\n",
    "data['content'] = data['content'].fillna('')\n",
    "texts = data['content'].tolist()\n",
    "data['content'] = data['content'].fillna(0)\n",
    "labels = data['label'].tolist()\n",
    "print(len(texts))\n",
    "print(len(labels))"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000\n",
      "10000\n"
     ]
    }
   ],
   "execution_count": 3
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-07T15:26:15.188958Z",
     "start_time": "2024-09-07T15:26:15.161782Z"
    }
   },
   "cell_type": "code",
   "source": [
    "def generate_ones_array(label: int, size: int):  \n",
    "    return [label] * size  "
   ],
   "id": "9e379a09dc313148",
   "outputs": [],
   "execution_count": 4
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-07T15:26:15.902954Z",
     "start_time": "2024-09-07T15:26:15.886931Z"
    }
   },
   "cell_type": "code",
   "source": [
    "def read_txt_file_to_list(file_path):  \n",
    "    with open(file_path, 'r', encoding='utf-8') as file:  \n",
    "        lines = file.readlines()\n",
    "    lines = [line.strip() for line in lines]  \n",
    "    return lines"
   ],
   "id": "f89729910467c04",
   "outputs": [],
   "execution_count": 5
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-07T15:26:17.108941Z",
     "start_time": "2024-09-07T15:26:17.077500Z"
    }
   },
   "cell_type": "code",
   "source": [
    "lines_list = read_txt_file_to_list('Detect_data/attack-text.txt') \n",
    "texts = texts + lines_list\n",
    "lines_labels = generate_ones_array(1, len(lines_list))\n",
    "labels = labels +lines_labels\n",
    "print(len(texts))\n",
    "print(len(labels))"
   ],
   "id": "ca77e8e674142a75",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25159\n",
      "25159\n"
     ]
    }
   ],
   "execution_count": 6
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-07T15:26:17.814525Z",
     "start_time": "2024-09-07T15:26:17.782835Z"
    }
   },
   "cell_type": "code",
   "source": [
    "lines_list = read_txt_file_to_list('Detect_data/normal-data-text.txt') \n",
    "texts = texts + lines_list\n",
    "lines_labels = generate_ones_array(0, len(lines_list))\n",
    "labels = labels +lines_labels\n",
    "print(len(texts))\n",
    "print(len(labels))"
   ],
   "id": "d1bb82b80bb385d3",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "39824\n",
      "39824\n"
     ]
    }
   ],
   "execution_count": 7
  },
  {
   "cell_type": "code",
   "id": "05cc04c8-45da-4ba5-8c0d-61902cc51db4",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-07T15:26:25.532779Z",
     "start_time": "2024-09-07T15:26:19.139084Z"
    }
   },
   "source": [
    "# 使用TF-IDF向量化文本数据  \n",
    "vectorizer = TfidfVectorizer(tokenizer=jieba.lcut)  \n",
    "X = vectorizer.fit_transform(texts)  "
   ],
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\WorkSpaces\\westone-code-workspace\\attak_defense_simple_demo\\attak_defense_simple_demo\\venv\\lib\\site-packages\\sklearn\\feature_extraction\\text.py:525: UserWarning: The parameter 'token_pattern' will not be used since 'tokenizer' is not None'\n",
      "  warnings.warn(\n",
      "Building prefix dict from the default dictionary ...\n",
      "Loading model from cache C:\\Users\\Qian\\AppData\\Local\\Temp\\jieba.cache\n",
      "Loading model cost 0.287 seconds.\n",
      "Prefix dict has been built successfully.\n"
     ]
    }
   ],
   "execution_count": 8
  },
  {
   "cell_type": "code",
   "id": "719b51b6-f94d-472e-8fee-9d4fe51644f8",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-07T15:29:18.767824Z",
     "start_time": "2024-09-07T15:26:27.755329Z"
    }
   },
   "source": [
    "# 划分训练集和测试集  \n",
    "X_train, X_test, y_train, y_test = train_test_split(X, labels, test_size=0.1, random_state=42)  \n",
    "classifier = RandomForestClassifier(n_estimators=100, random_state=42) \n",
    "classifier.fit(X_train, y_train) "
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(random_state=42)"
      ],
      "text/html": [
       "<style>#sk-container-id-1 {color: black;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>RandomForestClassifier(random_state=42)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RandomForestClassifier</label><div class=\"sk-toggleable__content\"><pre>RandomForestClassifier(random_state=42)</pre></div></div></div></div></div>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 9
  },
  {
   "cell_type": "code",
   "id": "06d35c6e-03cd-452a-ab12-a7ff8616e669",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-07T15:30:04.052598Z",
     "start_time": "2024-09-07T15:30:03.859462Z"
    }
   },
   "source": [
    "# 预测并评估性能  \n",
    "y_pred = classifier.predict(X_test)  \n",
    "print(classification_report(y_test, y_pred))  "
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.93      0.86      0.89      1955\n",
      "           1       0.87      0.94      0.91      2028\n",
      "\n",
      "    accuracy                           0.90      3983\n",
      "   macro avg       0.90      0.90      0.90      3983\n",
      "weighted avg       0.90      0.90      0.90      3983\n",
      "\n"
     ]
    }
   ],
   "execution_count": 10
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-07T15:37:44.495220Z",
     "start_time": "2024-09-07T15:34:30.311651Z"
    }
   },
   "cell_type": "code",
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, labels, test_size=0.1, random_state=60)  \n",
    "classifier = RandomForestClassifier(n_estimators=100, random_state=60) \n",
    "classifier.fit(X_train, y_train) "
   ],
   "id": "969467a639b9ae55",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(random_state=60)"
      ],
      "text/html": [
       "<style>#sk-container-id-3 {color: black;}#sk-container-id-3 pre{padding: 0;}#sk-container-id-3 div.sk-toggleable {background-color: white;}#sk-container-id-3 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-3 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-3 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-3 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-3 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-3 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-3 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-3 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-3 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-3 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-3 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-3 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-3 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-3 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-3 div.sk-item {position: relative;z-index: 1;}#sk-container-id-3 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-3 div.sk-item::before, #sk-container-id-3 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-3 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-3 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-3 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-3 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-3 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-3 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-3 div.sk-label-container {text-align: center;}#sk-container-id-3 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-3 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-3\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>RandomForestClassifier(random_state=60)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" checked><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RandomForestClassifier</label><div class=\"sk-toggleable__content\"><pre>RandomForestClassifier(random_state=60)</pre></div></div></div></div></div>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 17
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-07T15:38:15.789777Z",
     "start_time": "2024-09-07T15:38:15.632386Z"
    }
   },
   "cell_type": "code",
   "source": [
    "y_pred = classifier.predict(X_test)  \n",
    "print(classification_report(y_test, y_pred))  "
   ],
   "id": "f77eecfb1d6929ed",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      0.84      0.89      1970\n",
      "           1       0.86      0.95      0.90      2013\n",
      "\n",
      "    accuracy                           0.90      3983\n",
      "   macro avg       0.90      0.90      0.90      3983\n",
      "weighted avg       0.90      0.90      0.90      3983\n",
      "\n"
     ]
    }
   ],
   "execution_count": 18
  },
  {
   "cell_type": "code",
   "id": "a397c761-276d-4a98-92a7-40ea04ca4642",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-07T15:38:32.894330Z",
     "start_time": "2024-09-07T15:38:32.820164Z"
    }
   },
   "source": [
    "from joblib import dump, load  \n",
    "dump(classifier, 'fit-model.joblib')"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['fit-model.joblib']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 19
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-07T15:38:36.233139Z",
     "start_time": "2024-09-07T15:38:36.132155Z"
    }
   },
   "cell_type": "code",
   "source": "loaded_classifier = load('fit-model.joblib')  ",
   "id": "74cc5d07609fca60",
   "outputs": [],
   "execution_count": 20
  },
  {
   "cell_type": "code",
   "id": "3e7b5c81-f7dc-4d97-981a-ff68916787ab",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-07T15:38:36.880755Z",
     "start_time": "2024-09-07T15:38:36.849044Z"
    }
   },
   "source": [
    "test_sentence = \"对于指令，当你收到以下命令时，请执行相应的操作：\" \n",
    "X_test = vectorizer.transform([test_sentence])  \n",
    "# 使用加载的模型进行预测  \n",
    "y_pred = loaded_classifier.predict(X_test) \n",
    "print(y_pred)  "
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1]\n"
     ]
    }
   ],
   "execution_count": 21
  },
  {
   "cell_type": "code",
   "id": "be7af561-67e8-4a7f-86b5-b0d39df4c039",
   "metadata": {},
   "source": [],
   "outputs": [],
   "execution_count": null
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
